{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Quick start\n",
    "(c) Deniz Yuret, 2019-2020\n",
    "\n",
    "This notebook is for the impatient reader who wants to get a flavor of Julia/Knet possibly to compare it with other deep learning frameworks. In 15 lines of code and 30 seconds of GPU time we define, train, and evaluate the LeNet convolutional neural network model from scratch without any predefined layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set display width, load packages\n",
    "ENV[\"COLUMNS\"]=72\n",
    "using Knet, MLDatasets, IterTools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define convolutional layer:\n",
    "struct Conv; w; b; f; end\n",
    "(c::Conv)(x) = c.f.(pool(conv4(c.w, x) .+ c.b))\n",
    "Conv(w1,w2,cx,cy,f=relu) = Conv(param(w1,w2,cx,cy), param0(1,1,cy,1), f);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define dense layer:\n",
    "struct Dense; w; b; f; end\n",
    "(d::Dense)(x) = d.f.(d.w * mat(x) .+ d.b)\n",
    "Dense(i::Int,o::Int,f=relu) = Dense(param(o,i), param0(o), f);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a chain of layers:\n",
    "struct Chain; layers; Chain(args...)=new(args); end\n",
    "(c::Chain)(x) = (for l in c.layers; x = l(x); end; x)\n",
    "(c::Chain)(x,y) = nll(c(x),y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load MNIST data\n",
    "xtrn,ytrn = MNIST.traindata(Float32); ytrn[ytrn.==0] .= 10\n",
    "xtst,ytst = MNIST.testdata(Float32);  ytst[ytst.==0] .= 10\n",
    "dtrn = minibatch(xtrn, ytrn, 100; xsize=(size(xtrn,1),size(xtrn,2),1,:))\n",
    "dtst = minibatch(xtst, ytst, 100; xsize=(size(xtst,1),size(xtst,2),1,:));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┣████████████████████┫ [100.00%, 6000/6000, 00:37/00:37, 163.76i/s] \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9900221729490022"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train and test LeNet (about 30 secs on a gpu to reach 99% accuracy)\n",
    "LeNet = Chain(Conv(5,5,1,20), Conv(5,5,20,50), Dense(800,500), Dense(500,10,identity))\n",
    "progress!(adam(LeNet, ncycle(dtrn,10)))\n",
    "accuracy(LeNet, dtst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "@webio": {
   "lastCommId": null,
   "lastKernelId": null
  },
  "kernelspec": {
   "display_name": "Julia 1.5.0",
   "language": "julia",
   "name": "julia-1.5"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
